package com.jiuzhou.crawlers.taobao.crawler.taobao;
import javax.management.JMException;

import com.jiuzhou.crawlers.taobao.pipeline.taobao.TaoBaoPipeline;
import com.jiuzhou.crawlers.taobao.processor.taobao.TaoBaoProcessor;
import com.jiuzhou.crawlers.taobao.processor.taobao.constant.CommonConstant;

import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.monitor.SpiderMonitor;







public class TaoBaoStart  {

	
	
	public  static void main(String[] args) throws JMException {
		

		 CommonConstant.KEYWORD=""; //需要抓取商品的关键词, 注意：该关键词需要在淘宝搜索时有品牌
		CommonConstant.CURRENTTYPE="松下+"+CommonConstant.KEYWORD;  //抓取网页过程中，意外停止时的关键词
		  
		 //爬虫主要信息设置
		  String list = "https://s.taobao.com/search?q="+ CommonConstant.KEYWORD+"&";
		  Spider oschinaSpider = Spider.create(new TaoBaoProcessor())
		  .addUrl(list).addPipeline(new TaoBaoPipeline("hdfs://master:9000/user/yanganzhen/"+CommonConstant.KEYWORD+"/")).thread(20);
		  SpiderMonitor.instance().register(oschinaSpider);
		  oschinaSpider.start();
	}
	
	}


